{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import time\n",
    "import datetime\n",
    "import builtwith\n",
    "import whois\n",
    "import itertools\n",
    "import urllib.request\n",
    "import urllib.parse\n",
    "import urllib.robotparser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'web-servers': ['Nginx'],\n",
       " 'web-frameworks': ['Web2py', 'Twitter Bootstrap'],\n",
       " 'programming-languages': ['Python'],\n",
       " 'javascript-frameworks': ['jQuery', 'Modernizr', 'jQuery UI']}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "builtwith.parse('http://example.webscraping.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error: <urlopen error [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "builtwith.parse(\"http://www.google.com\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"domain_name\": [\n",
      "    \"EDU.COM\",\n",
      "    \"edu.com\"\n",
      "  ],\n",
      "  \"registrar\": \"HiChina Zhicheng Technology Ltd.\",\n",
      "  \"whois_server\": \"grs-whois.hichina.com\",\n",
      "  \"referral_url\": null,\n",
      "  \"updated_date\": \"2017-11-27 07:33:04\",\n",
      "  \"creation_date\": \"1998-11-24 05:00:00\",\n",
      "  \"expiration_date\": \"2025-11-23 05:00:00\",\n",
      "  \"name_servers\": [\n",
      "    \"DNS31.HICHINA.COM\",\n",
      "    \"DNS32.HICHINA.COM\"\n",
      "  ],\n",
      "  \"status\": [\n",
      "    \"clientTransferProhibited https://icann.org/epp#clientTransferProhibited\",\n",
      "    \"clientUpdateProhibited https://icann.org/epp#clientUpdateProhibited\"\n",
      "  ],\n",
      "  \"emails\": \"DomainAbuse@service.aliyun.com\",\n",
      "  \"dnssec\": \"unsigned\",\n",
      "  \"name\": null,\n",
      "  \"org\": null,\n",
      "  \"address\": null,\n",
      "  \"city\": \"Hangzhou\",\n",
      "  \"state\": \"Zhejiang\",\n",
      "  \"zipcode\": null,\n",
      "  \"country\": null\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(whois.whois(\"cqust.edu.com\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Throttle:\n",
    "    \"\"\"\n",
    "    Add a delay between downloads to the same domain\n",
    "    \"\"\"\n",
    "    def __init__(self,delay):\n",
    "        # amount of delay between downloads for each domain\n",
    "        self.delay = delay\n",
    "        # timestamp of when a domain was last accessed\n",
    "        self.domains = {}\n",
    "        \n",
    "    def wait(self,url):\n",
    "        domain = urllib.parse.urlparse(url).netloc\n",
    "        last_accessed = self.domains.get(domain)\n",
    "        if self.delay > 0 and last_accessed is not None:\n",
    "            sleep_secs = self.delay - (datetime.datetime.now() - last_accessed).seconds\n",
    "            if sleep_secs > 0:\n",
    "                # domain has been accessed recently so need to sleep\n",
    "                time.sleep(sleep_secs)\n",
    "        # update the last accessed time\n",
    "        self.domains[domain] = datetime.datetime.now()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def download(url,user_agent=\"wswp\",proxy=None,num_retries=2):\n",
    "    print('Downloading:',url)\n",
    "    headers = {'User-agent':user_agent}\n",
    "    request = urllib.request.Request(url,headers=headers)\n",
    "    opener = urllib.request.build_opener()\n",
    "    if proxy:\n",
    "        proxy_params = {urllib.parse.urlparse(url).scheme: proxy}\n",
    "        opener.add_handler(urllib.request.ProxyHandler(proxy_params))\n",
    "    try:\n",
    "        html = opener.open(request)\n",
    "    except urllib.request.HTTPError as e:\n",
    "        print('Download error:',e.reason)\n",
    "        html = None\n",
    "        if num_retries>0 and hasattr(e,'code') and 500<=e.code<600:\n",
    "            html = download(url,user_agent,proxy,num_retries-1)\n",
    "    return html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://example.webscraping.com/sitemap.xml\n",
      "252\n",
      "Downloading: http://example.webscraping.com/places/default/view/Afghanistan-1\n",
      "Downloading: http://example.webscraping.com/places/default/view/Aland-Islands-2\n",
      "Downloading: http://example.webscraping.com/places/default/view/Albania-3\n",
      "Downloading: http://example.webscraping.com/places/default/view/Algeria-4\n",
      "Downloading: http://example.webscraping.com/places/default/view/American-Samoa-5\n",
      "Downloading: http://example.webscraping.com/places/default/view/Andorra-6\n",
      "Downloading: http://example.webscraping.com/places/default/view/Angola-7\n",
      "Downloading: http://example.webscraping.com/places/default/view/Anguilla-8\n",
      "Downloading: http://example.webscraping.com/places/default/view/Antarctica-9\n",
      "Downloading: http://example.webscraping.com/places/default/view/Antigua-and-Barbuda-10\n",
      "Downloading: http://example.webscraping.com/places/default/view/Argentina-11\n",
      "Downloading: http://example.webscraping.com/places/default/view/Armenia-12\n",
      "Downloading: http://example.webscraping.com/places/default/view/Aruba-13\n",
      "Downloading: http://example.webscraping.com/places/default/view/Australia-14\n",
      "Downloading: http://example.webscraping.com/places/default/view/Austria-15\n",
      "Downloading: http://example.webscraping.com/places/default/view/Azerbaijan-16\n",
      "Downloading: http://example.webscraping.com/places/default/view/Bahamas-17\n",
      "Downloading: http://example.webscraping.com/places/default/view/Bahrain-18\n",
      "Downloading: http://example.webscraping.com/places/default/view/Bangladesh-19\n",
      "Downloading: http://example.webscraping.com/places/default/view/Barbados-20\n",
      "Downloading: http://example.webscraping.com/places/default/view/Belarus-21\n",
      "Downloading: http://example.webscraping.com/places/default/view/Belgium-22\n",
      "Downloading: http://example.webscraping.com/places/default/view/Belize-23\n",
      "Downloading: http://example.webscraping.com/places/default/view/Benin-24\n",
      "Downloading: http://example.webscraping.com/places/default/view/Bermuda-25\n",
      "Downloading: http://example.webscraping.com/places/default/view/Bhutan-26\n",
      "Downloading: http://example.webscraping.com/places/default/view/Bolivia-27\n",
      "Downloading: http://example.webscraping.com/places/default/view/Bonaire-Saint-Eustatius-and-Saba-28\n",
      "Downloading: http://example.webscraping.com/places/default/view/Bosnia-and-Herzegovina-29\n",
      "Downloading: http://example.webscraping.com/places/default/view/Botswana-30\n",
      "Downloading: http://example.webscraping.com/places/default/view/Bouvet-Island-31\n",
      "Downloading: http://example.webscraping.com/places/default/view/Brazil-32\n",
      "Downloading: http://example.webscraping.com/places/default/view/British-Indian-Ocean-Territory-33\n",
      "Downloading: http://example.webscraping.com/places/default/view/British-Virgin-Islands-34\n",
      "Downloading: http://example.webscraping.com/places/default/view/Brunei-35\n",
      "Downloading: http://example.webscraping.com/places/default/view/Bulgaria-36\n",
      "Downloading: http://example.webscraping.com/places/default/view/Burkina-Faso-37\n",
      "Downloading: http://example.webscraping.com/places/default/view/Burundi-38\n",
      "Downloading: http://example.webscraping.com/places/default/view/Cambodia-39\n",
      "Downloading: http://example.webscraping.com/places/default/view/Cameroon-40\n",
      "Downloading: http://example.webscraping.com/places/default/view/Canada-41\n",
      "Downloading: http://example.webscraping.com/places/default/view/Cape-Verde-42\n",
      "Downloading: http://example.webscraping.com/places/default/view/Cayman-Islands-43\n",
      "Downloading: http://example.webscraping.com/places/default/view/Central-African-Republic-44\n",
      "Downloading: http://example.webscraping.com/places/default/view/Chad-45\n",
      "Downloading: http://example.webscraping.com/places/default/view/Chile-46\n",
      "Downloading: http://example.webscraping.com/places/default/view/China-47\n",
      "Downloading: http://example.webscraping.com/places/default/view/Christmas-Island-48\n",
      "Downloading: http://example.webscraping.com/places/default/view/Cocos-Islands-49\n",
      "Downloading: http://example.webscraping.com/places/default/view/Colombia-50\n",
      "Downloading: http://example.webscraping.com/places/default/view/Comoros-51\n",
      "Downloading: http://example.webscraping.com/places/default/view/Cook-Islands-52\n",
      "Downloading: http://example.webscraping.com/places/default/view/Costa-Rica-53\n",
      "Downloading: http://example.webscraping.com/places/default/view/Croatia-54\n",
      "Downloading: http://example.webscraping.com/places/default/view/Cuba-55\n",
      "Downloading: http://example.webscraping.com/places/default/view/Curacao-56\n",
      "Downloading: http://example.webscraping.com/places/default/view/Cyprus-57\n",
      "Downloading: http://example.webscraping.com/places/default/view/Czech-Republic-58\n",
      "Downloading: http://example.webscraping.com/places/default/view/Democratic-Republic-of-the-Congo-59\n",
      "Downloading: http://example.webscraping.com/places/default/view/Denmark-60\n",
      "Downloading: http://example.webscraping.com/places/default/view/Djibouti-61\n",
      "Downloading: http://example.webscraping.com/places/default/view/Dominica-62\n",
      "Downloading: http://example.webscraping.com/places/default/view/Dominican-Republic-63\n",
      "Downloading: http://example.webscraping.com/places/default/view/East-Timor-64\n",
      "Downloading: http://example.webscraping.com/places/default/view/Ecuador-65\n",
      "Downloading: http://example.webscraping.com/places/default/view/Egypt-66\n",
      "Downloading: http://example.webscraping.com/places/default/view/El-Salvador-67\n",
      "Downloading: http://example.webscraping.com/places/default/view/Equatorial-Guinea-68\n",
      "Downloading: http://example.webscraping.com/places/default/view/Eritrea-69\n",
      "Downloading: http://example.webscraping.com/places/default/view/Estonia-70\n",
      "Downloading: http://example.webscraping.com/places/default/view/Ethiopia-71\n",
      "Downloading: http://example.webscraping.com/places/default/view/Falkland-Islands-72\n",
      "Downloading: http://example.webscraping.com/places/default/view/Faroe-Islands-73\n",
      "Downloading: http://example.webscraping.com/places/default/view/Fiji-74\n",
      "Downloading: http://example.webscraping.com/places/default/view/Finland-75\n",
      "Downloading: http://example.webscraping.com/places/default/view/France-76\n",
      "Downloading: http://example.webscraping.com/places/default/view/French-Guiana-77\n",
      "Downloading: http://example.webscraping.com/places/default/view/French-Polynesia-78\n",
      "Downloading: http://example.webscraping.com/places/default/view/French-Southern-Territories-79\n",
      "Downloading: http://example.webscraping.com/places/default/view/Gabon-80\n",
      "Downloading: http://example.webscraping.com/places/default/view/Gambia-81\n",
      "Downloading: http://example.webscraping.com/places/default/view/Georgia-82\n",
      "Downloading: http://example.webscraping.com/places/default/view/Germany-83\n",
      "Downloading: http://example.webscraping.com/places/default/view/Ghana-84\n",
      "Downloading: http://example.webscraping.com/places/default/view/Gibraltar-85\n",
      "Downloading: http://example.webscraping.com/places/default/view/Greece-86\n",
      "Downloading: http://example.webscraping.com/places/default/view/Greenland-87\n",
      "Downloading: http://example.webscraping.com/places/default/view/Grenada-88\n",
      "Downloading: http://example.webscraping.com/places/default/view/Guadeloupe-89\n",
      "Downloading: http://example.webscraping.com/places/default/view/Guam-90\n",
      "Downloading: http://example.webscraping.com/places/default/view/Guatemala-91\n",
      "Downloading: http://example.webscraping.com/places/default/view/Guernsey-92\n",
      "Downloading: http://example.webscraping.com/places/default/view/Guinea-93\n",
      "Downloading: http://example.webscraping.com/places/default/view/Guinea-Bissau-94\n",
      "Downloading: http://example.webscraping.com/places/default/view/Guyana-95\n",
      "Downloading: http://example.webscraping.com/places/default/view/Haiti-96\n",
      "Downloading: http://example.webscraping.com/places/default/view/Heard-Island-and-McDonald-Islands-97\n",
      "Downloading: http://example.webscraping.com/places/default/view/Honduras-98\n",
      "Downloading: http://example.webscraping.com/places/default/view/Hong-Kong-99\n",
      "Downloading: http://example.webscraping.com/places/default/view/Hungary-100\n",
      "Downloading: http://example.webscraping.com/places/default/view/Iceland-101\n",
      "Downloading: http://example.webscraping.com/places/default/view/India-102\n",
      "Downloading: http://example.webscraping.com/places/default/view/Indonesia-103\n",
      "Downloading: http://example.webscraping.com/places/default/view/Iran-104\n",
      "Downloading: http://example.webscraping.com/places/default/view/Iraq-105\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://example.webscraping.com/places/default/view/Ireland-106\n",
      "Downloading: http://example.webscraping.com/places/default/view/Isle-of-Man-107\n",
      "Downloading: http://example.webscraping.com/places/default/view/Israel-108\n",
      "Downloading: http://example.webscraping.com/places/default/view/Italy-109\n",
      "Downloading: http://example.webscraping.com/places/default/view/Ivory-Coast-110\n",
      "Downloading: http://example.webscraping.com/places/default/view/Jamaica-111\n",
      "Downloading: http://example.webscraping.com/places/default/view/Japan-112\n",
      "Downloading: http://example.webscraping.com/places/default/view/Jersey-113\n",
      "Downloading: http://example.webscraping.com/places/default/view/Jordan-114\n",
      "Downloading: http://example.webscraping.com/places/default/view/Kazakhstan-115\n",
      "Downloading: http://example.webscraping.com/places/default/view/Kenya-116\n",
      "Downloading: http://example.webscraping.com/places/default/view/Kiribati-117\n",
      "Downloading: http://example.webscraping.com/places/default/view/Kosovo-118\n",
      "Downloading: http://example.webscraping.com/places/default/view/Kuwait-119\n",
      "Downloading: http://example.webscraping.com/places/default/view/Kyrgyzstan-120\n",
      "Downloading: http://example.webscraping.com/places/default/view/Laos-121\n",
      "Downloading: http://example.webscraping.com/places/default/view/Latvia-122\n",
      "Downloading: http://example.webscraping.com/places/default/view/Lebanon-123\n",
      "Downloading: http://example.webscraping.com/places/default/view/Lesotho-124\n",
      "Downloading: http://example.webscraping.com/places/default/view/Liberia-125\n",
      "Downloading: http://example.webscraping.com/places/default/view/Libya-126\n",
      "Downloading: http://example.webscraping.com/places/default/view/Liechtenstein-127\n",
      "Downloading: http://example.webscraping.com/places/default/view/Lithuania-128\n",
      "Downloading: http://example.webscraping.com/places/default/view/Luxembourg-129\n",
      "Downloading: http://example.webscraping.com/places/default/view/Macao-130\n",
      "Downloading: http://example.webscraping.com/places/default/view/Macedonia-131\n",
      "Downloading: http://example.webscraping.com/places/default/view/Madagascar-132\n",
      "Downloading: http://example.webscraping.com/places/default/view/Malawi-133\n",
      "Downloading: http://example.webscraping.com/places/default/view/Malaysia-134\n",
      "Downloading: http://example.webscraping.com/places/default/view/Maldives-135\n",
      "Downloading: http://example.webscraping.com/places/default/view/Mali-136\n",
      "Downloading: http://example.webscraping.com/places/default/view/Malta-137\n",
      "Downloading: http://example.webscraping.com/places/default/view/Marshall-Islands-138\n",
      "Downloading: http://example.webscraping.com/places/default/view/Martinique-139\n",
      "Downloading: http://example.webscraping.com/places/default/view/Mauritania-140\n",
      "Downloading: http://example.webscraping.com/places/default/view/Mauritius-141\n",
      "Downloading: http://example.webscraping.com/places/default/view/Mayotte-142\n",
      "Downloading: http://example.webscraping.com/places/default/view/Mexico-143\n",
      "Downloading: http://example.webscraping.com/places/default/view/Micronesia-144\n",
      "Downloading: http://example.webscraping.com/places/default/view/Moldova-145\n",
      "Downloading: http://example.webscraping.com/places/default/view/Monaco-146\n",
      "Downloading: http://example.webscraping.com/places/default/view/Mongolia-147\n",
      "Downloading: http://example.webscraping.com/places/default/view/Montenegro-148\n",
      "Downloading: http://example.webscraping.com/places/default/view/Montserrat-149\n",
      "Downloading: http://example.webscraping.com/places/default/view/Morocco-150\n",
      "Downloading: http://example.webscraping.com/places/default/view/Mozambique-151\n",
      "Downloading: http://example.webscraping.com/places/default/view/Myanmar-152\n",
      "Downloading: http://example.webscraping.com/places/default/view/Namibia-153\n",
      "Downloading: http://example.webscraping.com/places/default/view/Nauru-154\n",
      "Downloading: http://example.webscraping.com/places/default/view/Nepal-155\n",
      "Downloading: http://example.webscraping.com/places/default/view/Netherlands-156\n",
      "Downloading: http://example.webscraping.com/places/default/view/Netherlands-Antilles-157\n",
      "Downloading: http://example.webscraping.com/places/default/view/New-Caledonia-158\n",
      "Downloading: http://example.webscraping.com/places/default/view/New-Zealand-159\n",
      "Downloading: http://example.webscraping.com/places/default/view/Nicaragua-160\n",
      "Downloading: http://example.webscraping.com/places/default/view/Niger-161\n",
      "Downloading: http://example.webscraping.com/places/default/view/Nigeria-162\n",
      "Downloading: http://example.webscraping.com/places/default/view/Niue-163\n",
      "Downloading: http://example.webscraping.com/places/default/view/Norfolk-Island-164\n",
      "Downloading: http://example.webscraping.com/places/default/view/North-Korea-165\n",
      "Downloading: http://example.webscraping.com/places/default/view/Northern-Mariana-Islands-166\n",
      "Downloading: http://example.webscraping.com/places/default/view/Norway-167\n",
      "Downloading: http://example.webscraping.com/places/default/view/Oman-168\n",
      "Downloading: http://example.webscraping.com/places/default/view/Pakistan-169\n",
      "Downloading: http://example.webscraping.com/places/default/view/Palau-170\n",
      "Downloading: http://example.webscraping.com/places/default/view/Palestinian-Territory-171\n",
      "Downloading: http://example.webscraping.com/places/default/view/Panama-172\n",
      "Downloading: http://example.webscraping.com/places/default/view/Papua-New-Guinea-173\n",
      "Downloading: http://example.webscraping.com/places/default/view/Paraguay-174\n",
      "Downloading: http://example.webscraping.com/places/default/view/Peru-175\n",
      "Downloading: http://example.webscraping.com/places/default/view/Philippines-176\n",
      "Downloading: http://example.webscraping.com/places/default/view/Pitcairn-177\n",
      "Downloading: http://example.webscraping.com/places/default/view/Poland-178\n",
      "Downloading: http://example.webscraping.com/places/default/view/Portugal-179\n",
      "Downloading: http://example.webscraping.com/places/default/view/Puerto-Rico-180\n",
      "Downloading: http://example.webscraping.com/places/default/view/Qatar-181\n",
      "Downloading: http://example.webscraping.com/places/default/view/Republic-of-the-Congo-182\n",
      "Downloading: http://example.webscraping.com/places/default/view/Reunion-183\n",
      "Downloading: http://example.webscraping.com/places/default/view/Romania-184\n",
      "Downloading: http://example.webscraping.com/places/default/view/Russia-185\n",
      "Downloading: http://example.webscraping.com/places/default/view/Rwanda-186\n",
      "Downloading: http://example.webscraping.com/places/default/view/Saint-Barthelemy-187\n",
      "Downloading: http://example.webscraping.com/places/default/view/Saint-Helena-188\n",
      "Downloading: http://example.webscraping.com/places/default/view/Saint-Kitts-and-Nevis-189\n",
      "Downloading: http://example.webscraping.com/places/default/view/Saint-Lucia-190\n",
      "Downloading: http://example.webscraping.com/places/default/view/Saint-Martin-191\n",
      "Downloading: http://example.webscraping.com/places/default/view/Saint-Pierre-and-Miquelon-192\n",
      "Downloading: http://example.webscraping.com/places/default/view/Saint-Vincent-and-the-Grenadines-193\n",
      "Downloading: http://example.webscraping.com/places/default/view/Samoa-194\n",
      "Downloading: http://example.webscraping.com/places/default/view/San-Marino-195\n",
      "Downloading: http://example.webscraping.com/places/default/view/Sao-Tome-and-Principe-196\n",
      "Downloading: http://example.webscraping.com/places/default/view/Saudi-Arabia-197\n",
      "Downloading: http://example.webscraping.com/places/default/view/Senegal-198\n",
      "Downloading: http://example.webscraping.com/places/default/view/Serbia-199\n",
      "Downloading: http://example.webscraping.com/places/default/view/Serbia-and-Montenegro-200\n",
      "Downloading: http://example.webscraping.com/places/default/view/Seychelles-201\n",
      "Downloading: http://example.webscraping.com/places/default/view/Sierra-Leone-202\n",
      "Downloading: http://example.webscraping.com/places/default/view/Singapore-203\n",
      "Downloading: http://example.webscraping.com/places/default/view/Sint-Maarten-204\n",
      "Downloading: http://example.webscraping.com/places/default/view/Slovakia-205\n",
      "Downloading: http://example.webscraping.com/places/default/view/Slovenia-206\n",
      "Downloading: http://example.webscraping.com/places/default/view/Solomon-Islands-207\n",
      "Downloading: http://example.webscraping.com/places/default/view/Somalia-208\n",
      "Downloading: http://example.webscraping.com/places/default/view/South-Africa-209\n",
      "Downloading: http://example.webscraping.com/places/default/view/South-Georgia-and-the-South-Sandwich-Islands-210\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://example.webscraping.com/places/default/view/South-Korea-211\n",
      "Downloading: http://example.webscraping.com/places/default/view/South-Sudan-212\n",
      "Downloading: http://example.webscraping.com/places/default/view/Spain-213\n",
      "Downloading: http://example.webscraping.com/places/default/view/Sri-Lanka-214\n",
      "Downloading: http://example.webscraping.com/places/default/view/Sudan-215\n",
      "Downloading: http://example.webscraping.com/places/default/view/Suriname-216\n",
      "Downloading: http://example.webscraping.com/places/default/view/Svalbard-and-Jan-Mayen-217\n",
      "Downloading: http://example.webscraping.com/places/default/view/Swaziland-218\n",
      "Downloading: http://example.webscraping.com/places/default/view/Sweden-219\n",
      "Downloading: http://example.webscraping.com/places/default/view/Switzerland-220\n",
      "Downloading: http://example.webscraping.com/places/default/view/Syria-221\n",
      "Downloading: http://example.webscraping.com/places/default/view/Taiwan-222\n",
      "Downloading: http://example.webscraping.com/places/default/view/Tajikistan-223\n",
      "Downloading: http://example.webscraping.com/places/default/view/Tanzania-224\n",
      "Downloading: http://example.webscraping.com/places/default/view/Thailand-225\n",
      "Downloading: http://example.webscraping.com/places/default/view/Togo-226\n",
      "Downloading: http://example.webscraping.com/places/default/view/Tokelau-227\n",
      "Downloading: http://example.webscraping.com/places/default/view/Tonga-228\n",
      "Downloading: http://example.webscraping.com/places/default/view/Trinidad-and-Tobago-229\n",
      "Downloading: http://example.webscraping.com/places/default/view/Tunisia-230\n",
      "Downloading: http://example.webscraping.com/places/default/view/Turkey-231\n",
      "Downloading: http://example.webscraping.com/places/default/view/Turkmenistan-232\n",
      "Downloading: http://example.webscraping.com/places/default/view/Turks-and-Caicos-Islands-233\n",
      "Downloading: http://example.webscraping.com/places/default/view/Tuvalu-234\n",
      "Downloading: http://example.webscraping.com/places/default/view/US-Virgin-Islands-235\n",
      "Downloading: http://example.webscraping.com/places/default/view/Uganda-236\n",
      "Downloading: http://example.webscraping.com/places/default/view/Ukraine-237\n",
      "Downloading: http://example.webscraping.com/places/default/view/United-Arab-Emirates-238\n",
      "Downloading: http://example.webscraping.com/places/default/view/United-Kingdom-239\n",
      "Downloading: http://example.webscraping.com/places/default/view/United-States-240\n",
      "Downloading: http://example.webscraping.com/places/default/view/United-States-Minor-Outlying-Islands-241\n",
      "Downloading: http://example.webscraping.com/places/default/view/Uruguay-242\n",
      "Downloading: http://example.webscraping.com/places/default/view/Uzbekistan-243\n",
      "Downloading: http://example.webscraping.com/places/default/view/Vanuatu-244\n",
      "Downloading: http://example.webscraping.com/places/default/view/Vatican-245\n",
      "Downloading: http://example.webscraping.com/places/default/view/Venezuela-246\n",
      "Downloading: http://example.webscraping.com/places/default/view/Vietnam-247\n",
      "Downloading: http://example.webscraping.com/places/default/view/Wallis-and-Futuna-248\n",
      "Downloading: http://example.webscraping.com/places/default/view/Western-Sahara-249\n",
      "Downloading: http://example.webscraping.com/places/default/view/Yemen-250\n",
      "Downloading: http://example.webscraping.com/places/default/view/Zambia-251\n",
      "Downloading: http://example.webscraping.com/places/default/view/Zimbabwe-252\n"
     ]
    }
   ],
   "source": [
    "def crawl_sitemap(url,delay=1):\n",
    "    throttle = Throttle(delay)\n",
    "    sitemap = download(url).read().decode('utf-8')\n",
    "    #extract the sitemap links\n",
    "    links = re.findall('<loc>(.*?)</loc>',sitemap)\n",
    "    #download each link\n",
    "    print(len(links))\n",
    "    for link in links:\n",
    "        throttle.wait(url)\n",
    "        html = download(link)\n",
    "# crawl_sitemap('http://example.webscraping.com/sitemap.xml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def crawl_ID(host,delay=1):\n",
    "    throttle = Throttle(delay)\n",
    "    # maximum number of consecutive download errors allowed\n",
    "    max_errors = 5\n",
    "    #current number of consecutive download errors\n",
    "    num_errors = 0\n",
    "    for page in itertools.count(1):\n",
    "        url = host + '/-%d' % page\n",
    "        throttle.wait(url)\n",
    "        html = download(url)\n",
    "        if html is None:\n",
    "            # recevied an error trying to download this webpage\n",
    "            num_errors += 1\n",
    "            if num_errors == max_errors:\n",
    "                # reached maximum number of consecutive errors so exit\n",
    "                break;\n",
    "        else:\n",
    "            # success - can scrape the result\n",
    "            num_errors = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def link_crawler(send_url,link_regex,delay=1,max_depth=2):\n",
    "    \"\"\"\n",
    "    Crawl from the given seed URL following links matched by link_regex\n",
    "    \"\"\"\n",
    "    throttle = Throttle(delay)\n",
    "    rp = urllib.robotparser.RobotFileParser()\n",
    "    rp.set_url(urllib.parse.urljoin(send_url,'/robots.txt'))\n",
    "    rp.read()\n",
    "    user_agent = \"Baiduspider\"\n",
    "    crawl_queue = [send_url]\n",
    "    seen = {send_url:0}\n",
    "    while crawl_queue:\n",
    "        url = crawl_queue.pop()\n",
    "        depth = seen[url]\n",
    "        if depth != max_depth:\n",
    "            # chec url passes robots.txt restrictions\n",
    "            if rp.can_fetch(user_agent,url):\n",
    "                throttle.wait(url)\n",
    "                html = download(url,user_agent,None,2)\n",
    "                if html == None:\n",
    "                    continue;\n",
    "                # filter for links matching our regular expression\n",
    "                for link in get_links(html.read().decode('utf-8')):\n",
    "                    # print('Check link_regex:',link)\n",
    "                    if re.match(link_regex,link):\n",
    "                        # from absolute link\n",
    "                        link = urllib.parse.urljoin(send_url,link)\n",
    "                        # check if have alreday seen this link\n",
    "                        if link not in seen:\n",
    "                            seen[link] = depth + 1\n",
    "                            crawl_queue.append(link)\n",
    "            else:\n",
    "                print('Blocked by robots.txt:',url)\n",
    "def get_links(html):\n",
    "    \"\"\"\n",
    "    Return a list of links from html\n",
    "    \"\"\"\n",
    "    # a regular expression to extract all links from the webpage\n",
    "    webpage_regex = re.compile('<a[^>]+href=[\"\\'](.*?)[\"\\']',re.IGNORECASE)\n",
    "    # list of all links from the webpaeg\n",
    "    return webpage_regex.findall(html)\n",
    "# link_crawler('http://example.webscraping.com','/places/default/(view|index)')\n",
    "# link_crawler('http://httpstat.us/500','/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://www.iqiyi.com\n",
      "Downloading: http://www.iqiyi.com/common/announce.html\n",
      "Downloading: http://www.iqiyi.com/common/secret.html\n",
      "Downloading: http://www.iqiyi.com/user/register/protocol.html\n",
      "Downloading: http://www.iqiyi.com/common/copyright.html?entry=iqiyi\n",
      "Downloading: http://help.iqiyi.com/?entry=homefooter\n",
      "Downloading: http://www.iqiyi.com/hezuo.html\n",
      "Downloading: http://mp.iqiyi.com/\n",
      "Downloading: http://www.iqiyi.com/common/wangdahz.html\n",
      "Downloading: http://open.iqiyi.com/\n",
      "Downloading: http://labs.iqiyi.com\n",
      "Downloading: http://zhaopin.iqiyi.com\n",
      "Downloading: http://www.iqiyi.com/common/bfljs.html\n",
      "Downloading: http://www.iqiyi.com/common/contactus.html\n",
      "Downloading: http://ir.iqiyi.com\n",
      "Downloading: http://www.iqiyi.com/common/aboutus.html\n",
      "Downloading: http://www.hd315.gov.cn/beian/view.asp?bianhao=010202012091700001\n"
     ]
    },
    {
     "ename": "UnicodeDecodeError",
     "evalue": "'utf-8' codec can't decode byte 0xbe in position 29: invalid start byte",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mUnicodeDecodeError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-efe3a7f78f40>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mlink_crawler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'http://www.iqiyi.com'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'/'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-9-401d81cd3b52>\u001b[0m in \u001b[0;36mlink_crawler\u001b[1;34m(send_url, link_regex, delay, max_depth)\u001b[0m\n\u001b[0;32m     21\u001b[0m                     \u001b[1;32mcontinue\u001b[0m\u001b[1;33m;\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m                 \u001b[1;31m# filter for links matching our regular expression\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m                 \u001b[1;32mfor\u001b[0m \u001b[0mlink\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mget_links\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhtml\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'utf-8'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m                     \u001b[1;31m# print('Check link_regex:',link)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[0mre\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlink_regex\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mlink\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mUnicodeDecodeError\u001b[0m: 'utf-8' codec can't decode byte 0xbe in position 29: invalid start byte"
     ]
    }
   ],
   "source": [
    "link_crawler('http://www.iqiyi.com','/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
